import requests
from lxml import etree
import pymongo

client = pymongo.MongoClient('localhost', 27017)
db = client.get_database("scrape")
collections = db.get_collection("words")

first_url = 'https://quotes.toscrape.com/'
def req(url):
    response = requests.get(url)
    text = response.text
    return text


def get_info():
    text = req(first_url)
    tree = etree.HTML(text)
    next_url = "https://quotes.toscrape.com"+tree.xpath("//div[@class='container']/div[@class='row']/div[@class='col-md-8']/nav/ul[@class='pager']/li[@class='next']/a/@href")[0]
    contents =  tree.xpath("//div[@class='container']/div[@class='row']/div[@class='col-md-8']/div[@class='quote']/span[@class='text']")
    authors = tree.xpath("//div[@class='container']/div[@class='row']/div[@class='col-md-8']/div[@class='quote']/span/small[@class='author']")
    for i in range(len(contents)):
        print(contents[i].text+"--"+authors[i].text)
    for i in range(5):
        text = req(next_url)
        tree = etree.HTML(text)
        contents = tree.xpath("//div[@class='container']/div[@class='row']/div[@class='col-md-8']/div[@class='quote']/span[@class='text']")
        authors = tree.xpath("//div[@class='container']/div[@class='row']/div[@class='col-md-8']/div[@class='quote']/span/small[@class='author']")
        for j in range(len(contents)):
            print(contents[j].text + "--" + authors[j].text)
        next_url = "https://quotes.toscrape.com" + tree.xpath("//div[@class='container']/div[@class='row']/div[@class='col-md-8']/nav/ul[@class='pager']/li[@class='next']/a/@href")[0]
if __name__ == '__main__':
    get_info()